{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "290cffd9",
   "metadata": {},
   "source": [
    "### 标签数量分布"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13f74296",
   "metadata": {},
   "source": [
    "数据集：\n",
    "\n",
    "sentence label\n",
    "\n",
    "... ...\n",
    "\n",
    "考虑是否解决样本分布imbalance问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "744de594",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install seaborn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ffd7b30",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.style.use('fivethirtyeight')\n",
    "\n",
    "train_data = pd.read_csv('data/train.csv', sep='\\t')\n",
    "test_data = pd.read_csv('data/dev.csv', sep='\\t')\n",
    "\n",
    "# 获得训练数据标签数量分布\n",
    "sns.countplot(\"label\", data=train_data)\n",
    "plt.title(\"train_data\")\n",
    "plt.show()\n",
    "\n",
    "# 获取验证数据标签数量分布\n",
    "sns.countplot(\"label\", data=test_data)\n",
    "plt.title(\"test_data\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8ebc542",
   "metadata": {},
   "source": [
    "### 句子长度分布"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef2f157e",
   "metadata": {},
   "source": [
    "有助于确定样本对齐长度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5087568d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data['sentence_length'] = list(map(lambda x: len(x), train_data['sentence']))\n",
    "\n",
    "# 绘制句子长度列的数量分布图\n",
    "sns.countplot(\"sentence_length\", data=train_data)\n",
    "# 主要关注count长度分布的纵坐标，不需要绘制横坐标\n",
    "plt.xticks([])\n",
    "plt.show()\n",
    "\n",
    "# 绘制dist长度分布图\n",
    "sns.distplot(train_data['sentence_length'])\n",
    "# 主要关注dist长度分布横坐标，不需要绘制纵坐标\n",
    "plt.yticks([])\n",
    "plt.show()\n",
    "\n",
    "# 验证集同理..."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdb0d34e",
   "metadata": {},
   "source": [
    "### 正负样本长度散点分布"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c9c9810",
   "metadata": {},
   "source": [
    "有助于确定异常点"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba5a3c45",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.stripplot(y='sentence_length', x='label', data=train_data)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34528803",
   "metadata": {},
   "source": [
    "### 词汇数目"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f920ae76",
   "metadata": {},
   "source": [
    "用于构建word embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcf2bc49",
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "from itertools import chain\n",
    "\n",
    "train_vocab = set(chain(*map(lambda x: jieba.lcut(x), train_data['sentence'])))\n",
    "print(\"训练集供包含不同词汇总数为：\", len(train_vocab))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36540c75",
   "metadata": {},
   "source": [
    "### 获取正负样本的高频形容词词云"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f22bb5b8",
   "metadata": {},
   "source": [
    "从语言学角度出发，统计形容词有助于完成最终任务"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f09e73c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba.posseg as pseg\n",
    "\n",
    "def get_a_list(text):\n",
    "    r = []\n",
    "    for g in pseg.lcut(text):\n",
    "        if g.flag == 'a':\n",
    "            r.appendp(g.word)\n",
    "    return r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd8d57c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install wordcloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b0c93fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from wordcloud import WordCloud\n",
    "\n",
    "def get_word_cloud(words_list):\n",
    "    word_cloud = WordCloud(font_path='./SimHei.ttf', max_words=100, background_color='white')\n",
    "    words_string = ' '.join(words_list)  # 词云生成器需要字符串形式\n",
    "    word_cloud.generate(words_string)\n",
    "    \n",
    "    plt.figure()\n",
    "    plt.imshow(word_cloud, interpolation='bilinear')\n",
    "    plt.axis('off')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09a73b7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获得训练集上正样本\n",
    "p_train_data = train_data[train_data[\"label\"] == 1][\"sentence\"]\n",
    "train_p_a_vocab = chain(*map(lambda x: get_a_list(x), p_train_data))  # 获得每个句子的形容词\n",
    "get_word_cloud(train_p_a_vocab)  # 绘制词云"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29a3787a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torchX",
   "language": "python",
   "name": "torchx"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
